from __future__ import division
from operator import itemgetter, attrgetter

import gc
import math
import matplotlib
import os
import pylab
import random
import sys
import time
from sets import Set
from scipy import stats
import numpy as np

# TODO: cross check about the 2D and the 1D
top10PostingClassificationWithNumOfPostingsDict = {}
relatedPostingClassificationWithNumOfPostingsDict = {}

print "Program Begins..."

inputFileName = "/home/diaosi/gov2ClearYourMindAndDoItAgain/impactWithLengthAnalyze20131130Night/probabiliesBasedOnNumeratorANDDenominator"
inputFileHandler = open(inputFileName,"r")
currentLine = inputFileHandler.readline()
currentLine = inputFileHandler.readline()
for line in inputFileHandler.readlines():
    lineElements = line.strip().split(" ")
    if len(lineElements) == 6:
        currentImpactScoreClassLabel = int(lineElements[-4])
        currentTOP10Postings = int( lineElements[-3] )
        currentRelatedPostings = int( lineElements[-2] )
        
        if currentImpactScoreClassLabel not in top10PostingClassificationWithNumOfPostingsDict or currentImpactScoreClassLabel not in relatedPostingClassificationWithNumOfPostingsDict:
            top10PostingClassificationWithNumOfPostingsDict[currentImpactScoreClassLabel] = 0
            relatedPostingClassificationWithNumOfPostingsDict[currentImpactScoreClassLabel] = 0
        
        top10PostingClassificationWithNumOfPostingsDict[currentImpactScoreClassLabel] += currentTOP10Postings
        relatedPostingClassificationWithNumOfPostingsDict[currentImpactScoreClassLabel] += currentRelatedPostings

print "len(top10PostingClassificationWithNumOfPostingsDict): ",len(top10PostingClassificationWithNumOfPostingsDict)
print "len(relatedPostingClassificationWithNumOfPostingsDict): ",len(relatedPostingClassificationWithNumOfPostingsDict)

totalNumOfRelatedPostings = 0
totalNumOfTop10Postings = 0

class1NumOfRelatedPostings = 0
class1NumOfTop10Postings = 0

class2NumOfRelatedPostings = 0
class2NumOfTop10Postings = 0

for i in range(0,16):
    class1NumOfRelatedPostings += relatedPostingClassificationWithNumOfPostingsDict[i]
    class1NumOfTop10Postings += top10PostingClassificationWithNumOfPostingsDict[i]
    totalNumOfRelatedPostings += relatedPostingClassificationWithNumOfPostingsDict[i]
    totalNumOfTop10Postings += top10PostingClassificationWithNumOfPostingsDict[i]
print "0",class1NumOfTop10Postings,class1NumOfRelatedPostings

for i in range(16,18):
    class2NumOfRelatedPostings += relatedPostingClassificationWithNumOfPostingsDict[i]
    class2NumOfTop10Postings += top10PostingClassificationWithNumOfPostingsDict[i]
    totalNumOfRelatedPostings += relatedPostingClassificationWithNumOfPostingsDict[i]
    totalNumOfTop10Postings += top10PostingClassificationWithNumOfPostingsDict[i]
print "16",class2NumOfTop10Postings,class2NumOfRelatedPostings


for i in range(18,len(top10PostingClassificationWithNumOfPostingsDict)):
    totalNumOfRelatedPostings += relatedPostingClassificationWithNumOfPostingsDict[i]
    totalNumOfTop10Postings += top10PostingClassificationWithNumOfPostingsDict[i]
    print i,top10PostingClassificationWithNumOfPostingsDict[i],relatedPostingClassificationWithNumOfPostingsDict[i]

print "totalNumOfRelatedPostings:",totalNumOfRelatedPostings
print "totalNumOfTop10Postings:",totalNumOfTop10Postings
inputFileHandler.close()
exit(1)


'''
# x = []
# y = []
# yRaw = []

inputFileName = "/home/diaosi/gov2ClearYourMindAndDoItAgain/partialBM25ScoreBucketingPrelimanryResultsV3_SMOOTHED_AND_ready_for_REGRESSION_MODIFICATION.csv"
inputFileHandler = open(inputFileName,"r")
inputFileHandler.readline()
totalNumOfPostings = 0
totalNumOfTop10Postings = 0
currentNumOfPostings = 0
currentNumOfTop10Postings = 0
for line in inputFileHandler.readlines():
    lineElements = line.strip().split("\t")
    if lineElements[-3].strip() != "sameAbove":
        currentNumOfPostings = int( lineElements[-3].strip() )
        totalNumOfPostings += currentNumOfPostings
        currentNumOfTop10Postings = int( lineElements[-2].strip() )
        totalNumOfTop10Postings += currentNumOfTop10Postings
        impactScoreClassLabel = lineElements[-6].strip()
        impactScoreLowerBound = lineElements[-5].strip()
    if lineElements[-3].strip() != "sameAbove" and lineElements[-2].strip() != "sameAbove":
        finalProbability = 0.0
        if currentNumOfPostings == 0:
            finalProbability = 0.0
            print impactScoreClassLabel,impactScoreLowerBound,currentNumOfPostings,currentNumOfTop10Postings,finalProbability
        else:
            finalProbability = currentNumOfTop10Postings/currentNumOfPostings
            print impactScoreClassLabel,impactScoreLowerBound,currentNumOfPostings,currentNumOfTop10Postings,finalProbability
        # yRaw.append(finalProbability)

# for index,currentProbability in enumerate(yRaw):
#     x.append( index )
#     y.append( currentProbability )
# matplotlib.pyplot.scatter(x,y)
# matplotlib.pyplot.show()

print "totalNumOfPostings:",totalNumOfPostings
print "totalNumOfTop10Postings:",totalNumOfTop10Postings
inputFileHandler.close()
'''

print "Program Ends."    
exit(1)
